import tushare as ts
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import statsmodels.api as sm
from scipy import stats
from datetime import datetime, timedelta
from analysis_engine import SectorAnalyzer
import logging

logger = logging.getLogger(__name__)

class FactorValidator:
    def __init__(self):
        ts.set_token("8f0c9f36fea80fe9cc6a0d480a87ba58c48a91182d0f7562e9ac9706")
        self.pro = ts.pro_api()
        self.analyzer = SectorAnalyzer()
        # 使用同一个缓存管理器实例
        self.cache_manager = self.analyzer.cache_manager
        
    def get_future_returns(self, trade_date, n_days=5):
        """
        计算从trade_date开始未来n_days的行业收益率
        """
        # 首先检查预加载的数据
        preloaded_returns = self.cache_manager.get_preloaded_future_returns(trade_date, n_days)
        if preloaded_returns is not None:
            logger.debug(f"从预加载数据获取未来收益率: {trade_date}, {n_days}天")
            return preloaded_returns
            
        # 创建缓存文件名
        cache_key = f"future_returns_{trade_date}_{n_days}"
        cache_path = os.path.join(self.cache_manager.cache_dir, f"{cache_key}.csv")
        
        # 检查缓存是否存在
        if os.path.exists(cache_path):
            try:
                # 从缓存加载数据
                sector_returns = pd.read_csv(cache_path, index_col=0, squeeze=True)
                logger.debug(f"从缓存加载未来收益率数据: {cache_path}")
                # 将加载的数据添加到预加载缓存中
                self.cache_manager.preloaded_future_returns[(trade_date, n_days)] = sector_returns
                return sector_returns
            except Exception as e:
                logger.error(f"加载未来收益率缓存失败: {str(e)}")
                # 如果加载失败，继续执行获取数据的逻辑
        
        # 获取未来交易日
        end_date = datetime.strptime(trade_date, '%Y%m%d')
        future_date = end_date + timedelta(days=n_days*2)  # 留出足够的日历日以获取足够的交易日
        trade_dates = self.analyzer.get_trade_dates(trade_date, future_date.strftime('%Y%m%d'))
        
        if len(trade_dates) <= n_days:
            print(f"无法获取足够的未来交易日数据")
            return None
            
        future_date = trade_dates[n_days]  # 获取未来第n个交易日
        
        # 获取当前日期和未来日期的行业指数数据
        # 这里假设使用申万一级行业指数，实际使用时可能需要调整
        try:
            # 获取当前日期的行业数据
            current_data = self.analyzer.get_stock_data(trade_date)
            future_data = self.analyzer.get_stock_data(future_date)
            
            # 计算行业收益率
            # 这里简化处理，实际应该使用行业指数或成分股加权收益
            # 获取当前日期和未来日期的股票价格
            current_prices = {}
            future_prices = {}
            
            # 获取当前日期的股票价格 - 检查缓存
            current_price_cache = os.path.join(self.cache_manager.cache_dir, f"daily_prices_{trade_date}.csv")
            future_price_cache = os.path.join(self.cache_manager.cache_dir, f"daily_prices_{future_date}.csv")
            
            stocks = list(set(current_data['ts_code']))
            if len(stocks) > 0:
                # 尝试从缓存加载当前日期价格数据
                if os.path.exists(current_price_cache):
                    try:
                        current_price_data = pd.read_csv(current_price_cache)
                        logger.debug(f"从缓存加载当日价格数据: {current_price_cache}")
                    except Exception:
                        # 如果加载失败，从API获取
                        current_price_data = self.pro.daily(trade_date=trade_date, fields='ts_code,close')
                        # 保存到缓存
                        current_price_data.to_csv(current_price_cache, index=False)
                else:
                    # 从API获取并保存到缓存
                    current_price_data = self.pro.daily(trade_date=trade_date, fields='ts_code,close')
                    current_price_data.to_csv(current_price_cache, index=False)
                
                current_prices = dict(zip(current_price_data['ts_code'], current_price_data['close']))
                
                # 尝试从缓存加载未来日期价格数据
                if os.path.exists(future_price_cache):
                    try:
                        future_price_data = pd.read_csv(future_price_cache)
                        logger.debug(f"从缓存加载未来价格数据: {future_price_cache}")
                    except Exception:
                        # 如果加载失败，从API获取
                        future_price_data = self.pro.daily(trade_date=future_date, fields='ts_code,close')
                        # 保存到缓存
                        future_price_data.to_csv(future_price_cache, index=False)
                else:
                    # 从API获取并保存到缓存
                    future_price_data = self.pro.daily(trade_date=future_date, fields='ts_code,close')
                    future_price_data.to_csv(future_price_cache, index=False)
                
                future_prices = dict(zip(future_price_data['ts_code'], future_price_data['close']))
            
            # 计算每个行业的收益率
            sector_returns = {}
            for sector in current_data['sw_industry'].unique():
                sector_stocks = current_data[current_data['sw_industry'] == sector]['ts_code']
                sector_returns_list = []
                
                for stock in sector_stocks:
                    if stock in current_prices and stock in future_prices:
                        stock_return = (future_prices[stock] / current_prices[stock]) - 1
                        sector_returns_list.append(stock_return)
                
                if sector_returns_list:
                    # 使用等权平均计算行业收益
                    sector_returns[sector] = np.mean(sector_returns_list)
                else:
                    sector_returns[sector] = np.nan
            
            # 将结果保存到缓存
            sector_returns_series = pd.Series(sector_returns)
            sector_returns_series.to_csv(cache_path)
            logger.debug(f"未来收益率数据已缓存: {cache_path}")
            
            return sector_returns_series
            
        except Exception as e:
            print(f"计算未来收益率失败: {str(e)}")
            return None
    
    def calculate_ic(self, factor_scores, future_returns):
        """
        计算因子IC值（Information Coefficient）
        增强版IC计算，使用Spearman秩相关系数并添加稳健性检查
        """
        # 合并因子得分和未来收益
        if future_returns is None or len(future_returns) == 0:
            return None
            
        # 确保输入为Series类型
        if not isinstance(factor_scores, pd.Series):
            factor_scores = pd.Series(factor_scores)
        if not isinstance(future_returns, pd.Series):
            future_returns = pd.Series(future_returns)
            
        # 对齐索引并合并数据
        merged_data = pd.DataFrame({
            'factor': factor_scores,
            'returns': future_returns
        }).dropna()
        
        # 增强数据量检查
        if len(merged_data) < 5:  # 至少需要5个有效数据点
            return None
            
        # 检查因子得分和收益的方差
        if merged_data['factor'].std() < 1e-6 or merged_data['returns'].std() < 1e-6:
            return None
            
        try:
            # 计算Spearman秩相关系数，添加异常处理
            ic = merged_data['factor'].corr(merged_data['returns'], method='spearman')
            
            # 添加IC值合理性检查
            if np.isnan(ic) or abs(ic) > 1.0:
                return None
                
            return ic
        except Exception as e:
            print(f"计算IC值时发生错误: {str(e)}")
            return None
    
    def run_t_test(self, ic_series):
        """
        对IC序列进行t检验，判断是否显著大于0
        """
        if ic_series is None or len(ic_series) < 10:  # 至少需要10个IC值
            return None, None
            
        # 去除缺失值
        ic_series = ic_series.dropna()
        
        # 计算均值和标准差
        ic_mean = ic_series.mean()
        ic_std = ic_series.std()
        
        # 计算t值和p值
        t_stat = ic_mean / (ic_std / np.sqrt(len(ic_series)))
        p_value = 1 - stats.t.cdf(t_stat, len(ic_series)-1)
        
        return t_stat, p_value
    
    def get_factor_scores(self, factor_name, final_scores):
        """
        根据因子名称获取对应的因子得分
        
        Parameters:
        -----------
        factor_name : str
            因子名称
        final_scores : DataFrame
            包含所有因子得分的DataFrame
            
        Returns:
        --------
        pd.Series
            对应因子的得分序列
        """
        # 确保final_scores不为空
        if final_scores is None or final_scores.empty:
            return pd.Series()
            
        # 确保'sw_industry'列存在
        if 'sw_industry' not in final_scores.columns:
            return pd.Series()
            
        # 根据因子名称获取对应的得分列
        factor_column_map = {
            'density': 'density_score',
            'consecutive': 'consecutive_score',
            'historical': 'historical_score',
            'strength': 'strength_score'
        }
        
        # 获取对应的因子得分列名
        score_column = factor_column_map.get(factor_name, 'score')
        
        # 检查得分列是否存在
        if score_column not in final_scores.columns:
            return pd.Series()
            
        return final_scores.set_index('sw_industry')[score_column]
    
    def run_quantile_backtest(self, factor_name, start_date, end_date, n_groups=5, n_days=5, all_data=None):
        """
        按因子得分将行业分组，回测各组合的表现
        完全优化版本：使用内存中的数据进行计算，避免重复IO操作
        
        Parameters:
        -----------
        factor_name : str
            因子名称
        start_date : str
            开始日期
        end_date : str
            结束日期
        n_groups : int
            分组数量
        n_days : int
            未来收益计算天数
        all_data : dict
            预先加载的所有数据，如果为None则会重新加载
        """
        # 获取交易日序列
        trade_dates = self.analyzer.get_trade_dates(start_date, end_date)
        
        # 存储各组合的收益率序列
        group_returns = {i: [] for i in range(1, n_groups+1)}
        dates = []
        
        # 如果没有传入预加载的数据，则需要加载数据
        if all_data is None:
            print(f"没有传入预加载数据，开始加载分位数回测所需数据...")
            all_data = {
                'factor_scores': {},
                'future_returns': {}
            }
            
            # 检查是否有预加载的数据
            has_preloaded_data = len(self.cache_manager.preloaded_stock_data) > 0
            if not has_preloaded_data:
                print(f"预加载数据不存在，将加载所有必要数据")
                # 预加载所有必要的数据
                self.cache_manager.preload_stock_data(trade_dates)
                self.cache_manager.preload_future_returns(trade_dates, n_days)
            else:
                print(f"使用已预加载的数据进行分位数回测")
            
            # 在内存中计算所有日期的因子得分和未来收益率
            print("正在内存中计算所有交易日的因子得分和未来收益率...")
            for date in trade_dates:
                try:
                    # 获取股票数据
                    stock_data = self.cache_manager.get_preloaded_stock_data(date)
                    if stock_data is None:
                        stock_data = self.analyzer.get_stock_data(date)
                    
                    if stock_data is not None and not stock_data.empty:
                        # 计算因子得分
                        sector_stats, quantiles = self.analyzer.calculate_indicators(date)
                        final_scores = self.analyzer.calculate_scores(sector_stats, quantiles, stock_data)
                        
                        # 获取特定因子得分
                        factor_scores = self.get_factor_scores(factor_name, final_scores)
                        all_data['factor_scores'][date] = factor_scores
                    
                    # 获取未来收益率
                    future_returns = self.cache_manager.get_preloaded_future_returns(date, n_days)
                    if future_returns is None:
                        future_returns = self.get_future_returns(date, n_days)
                    if future_returns is not None:
                        all_data['future_returns'][date] = future_returns
                except Exception as e:
                    print(f"处理日期{date}数据失败: {str(e)}")
                    continue
            
            print(f"数据处理完成，成功加载 {len(all_data['factor_scores'])} 个交易日的因子得分和 {len(all_data['future_returns'])} 个交易日的未来收益率")
        else:
            print(f"使用传入的预加载数据进行分位数回测，共有 {len(all_data['factor_scores'])} 个交易日的数据")
        
        # 使用内存中的数据进行分组回测
        for date in trade_dates:
            try:
                if date in all_data['factor_scores'] and date in all_data['future_returns']:
                    factor_scores = all_data['factor_scores'][date]
                    future_returns = all_data['future_returns'][date]
                    
                    if len(factor_scores) > 0:
                        # 合并因子得分和未来收益
                        merged_data = pd.DataFrame({
                            'factor': factor_scores,
                            'returns': future_returns
                        }).dropna()
                        
                        if len(merged_data) >= n_groups:  # 确保有足够的数据进行分组
                            # 按因子得分分组
                            merged_data['group'] = pd.qcut(merged_data['factor'], n_groups, labels=range(1, n_groups+1))
                            
                            # 计算各组合的平均收益率
                            group_avg_returns = merged_data.groupby('group')['returns'].mean()
                            
                            # 存储各组合的收益率
                            for group, ret in group_avg_returns.items():
                                group_returns[group].append(ret)
                            
                            dates.append(date)
            except Exception as e:
                print(f"日期{date}分组回测失败: {str(e)}")
                continue
        
        # 计算累积收益率
        cumulative_returns = {}
        for group, returns in group_returns.items():
            if returns:  # 确保有收益率数据
                cumulative_returns[group] = (1 + pd.Series(returns)).cumprod() - 1
            else:
                cumulative_returns[group] = pd.Series([])
        
        return cumulative_returns, dates
    
    def run_parameter_sensitivity(self, factor_name, start_date, end_date, param_values, n_days=5, all_data=None):
        """
        参数敏感性测试
        完全优化版本：使用内存中的数据进行计算，避免重复IO操作
        
        Parameters:
        -----------
        factor_name : str
            因子名称
        start_date : str
            开始日期
        end_date : str
            结束日期
        param_values : list
            参数值列表
        n_days : int
            未来收益计算天数
        all_data : dict
            预先加载的所有数据，如果为None则会重新加载
        """
        # 存储不同参数下的IC值
        param_ic_values = {param: [] for param in param_values}
        
        # 获取交易日序列
        trade_dates = self.analyzer.get_trade_dates(start_date, end_date)
        test_dates = trade_dates[:30]  # 为了效率，只使用部分日期
        
        # 如果没有传入预加载的数据，则需要加载数据
        if all_data is None:
            print(f"没有传入预加载数据，开始加载参数敏感性测试所需数据...")
            all_data = {
                'sector_stats': {},
                'quantiles': {},
                'stock_data': {},
                'future_returns': {}
            }
            
            # 检查是否有预加载的数据
            has_preloaded_data = len(self.cache_manager.preloaded_stock_data) > 0
            if not has_preloaded_data:
                print(f"预加载数据不存在，将加载所有必要数据")
                # 预加载所有必要的数据
                self.cache_manager.preload_stock_data(test_dates)
                self.cache_manager.preload_future_returns(test_dates, n_days)
            else:
                print(f"使用已预加载的数据进行参数敏感性测试")
            
            # 在内存中计算所有日期的因子得分和未来收益率
            print("正在内存中计算所有测试日期的数据...")
            for date in test_dates:
                try:
                    # 获取股票数据
                    stock_data = self.cache_manager.get_preloaded_stock_data(date)
                    if stock_data is None:
                        stock_data = self.analyzer.get_stock_data(date)
                    if stock_data is not None and not stock_data.empty:
                        all_data['stock_data'][date] = stock_data
                    
                    # 获取行业指标数据
                    sector_stats, quantiles = self.analyzer.calculate_indicators(date)
                    all_data['sector_stats'][date] = sector_stats
                    all_data['quantiles'][date] = quantiles
                    
                    # 获取未来收益率
                    future_returns = self.cache_manager.get_preloaded_future_returns(date, n_days)
                    if future_returns is None:
                        future_returns = self.get_future_returns(date, n_days)
                    if future_returns is not None:
                        all_data['future_returns'][date] = future_returns
                except Exception as e:
                    print(f"处理日期{date}数据失败: {str(e)}")
                    continue
            
            print(f"数据处理完成，成功加载 {len(all_data['stock_data'])} 个交易日的股票数据")
            print(f"成功加载 {len(all_data['future_returns'])} 个交易日的未来收益率")
        else:
            print(f"使用传入的预加载数据进行参数敏感性测试")
        
        # 使用内存中的数据进行参数敏感性测试
        for date in test_dates:
            try:
                if date in all_data['future_returns'] and date in all_data['sector_stats'] and date in all_data['stock_data']:
                    future_returns = all_data['future_returns'][date]
                    sector_stats = all_data['sector_stats'][date]
                    quantiles = all_data['quantiles'][date]
                    stock_data = all_data['stock_data'][date]
                    
                    # 计算基础因子得分，只需计算一次
                    final_scores = self.analyzer.calculate_scores(sector_stats, quantiles, stock_data)
                    base_factor_scores = self.get_factor_scores(factor_name, final_scores)
                    
                    for param in param_values:
                        # 根据不同的因子和参数值调整因子得分
                        if factor_name == 'density':
                            # 调整涨停密度的权重
                            factor_scores = base_factor_scores * param
                        elif factor_name == 'historical':
                            # 调整历史分位值的阈值
                            factor_scores = base_factor_scores * param / 100
                        else:
                            # 其他因子的参数敏感性测试
                            factor_scores = base_factor_scores * param
                        
                        # 计算IC值
                        ic = self.calculate_ic(factor_scores, future_returns)
                        if ic is not None:
                            param_ic_values[param].append(ic)
            except Exception as e:
                print(f"日期{date}参数敏感性测试失败: {str(e)}")
                continue
        
        # 计算各参数下的平均IC值
        avg_ic_values = {param: np.mean(ics) if ics else np.nan for param, ics in param_ic_values.items()}
        
        return avg_ic_values
    
    def batch_process_data(self, trade_dates, n_days=5):
        """
        批量处理多个交易日的数据，一次性加载所有必要数据到内存
        
        Parameters:
        -----------
        trade_dates : list
            交易日期列表
        n_days : int, optional
            未来收益计算天数
            
        Returns:
        --------
        dict
            包含所有加载数据的字典
        """
        import concurrent.futures
        import threading
        
        # 创建数据容器
        all_data = {
            'stock_data': {},          # 股票数据
            'sector_stats': {},        # 行业统计数据
            'quantiles': {},           # 分位数数据
            'future_returns': {},      # 未来收益率
            'index_data': {}           # 指数数据
        }
        
        # 检查是否有预加载的数据
        has_preloaded_data = len(self.cache_manager.preloaded_stock_data) > 0
        if not has_preloaded_data:
            print("预加载数据不存在，开始批量加载所有必要数据...")
            # 批量预加载所有必要的数据
            self.cache_manager.preload_stock_data(trade_dates)
            self.cache_manager.preload_future_returns(trade_dates, n_days)
            self.cache_manager.preload_index_data(trade_dates)
        else:
            print(f"使用已预加载的数据进行处理")
        
        print("正在内存中并行处理所有交易日的数据...")
        
        # 创建线程锁，保护共享资源
        lock = threading.Lock()
        
        # 定义处理单个日期数据的函数
        def process_single_date(date):
            result = {}
            try:
                # 获取股票数据
                stock_data = self.cache_manager.get_preloaded_stock_data(date)
                if stock_data is None:
                    stock_data = self.analyzer.get_stock_data(date)
                if stock_data is not None and not stock_data.empty:
                    result['stock_data'] = stock_data
                
                # 获取行业指标数据
                sector_stats, quantiles = self.analyzer.calculate_indicators(date)
                result['sector_stats'] = sector_stats
                result['quantiles'] = quantiles
                
                # 获取未来收益率
                future_returns = self.cache_manager.get_preloaded_future_returns(date, n_days)
                if future_returns is None:
                    future_returns = self.get_future_returns(date, n_days)
                if future_returns is not None:
                    result['future_returns'] = future_returns
                
                # 获取指数数据
                index_data = self.cache_manager.get_preloaded_index_data(date)
                if index_data is None:
                    index_data = self.analyzer.get_index_data(date)
                if index_data is not None and not index_data.empty:
                    result['index_data'] = index_data.iloc[0]
                
                return date, result
            except Exception as e:
                print(f"处理日期{date}的数据失败: {str(e)}")
                return date, None
        
        # 使用线程池并行处理数据
        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
            futures = {executor.submit(process_single_date, date): date for date in trade_dates}
            for future in concurrent.futures.as_completed(futures):
                date, result = future.result()
                if result:
                    with lock:
                        for key, value in result.items():
                            if key in all_data and value is not None:
                                all_data[key][date] = value
        
        print(f"数据处理完成，共加载 {len(all_data['stock_data'])} 个交易日的股票数据")
        print(f"共加载 {len(all_data['future_returns'])} 个交易日的未来收益率")
        print(f"共加载 {len(all_data['sector_stats'])} 个交易日的行业统计数据")
        
        return all_data
    
    def validate_factor(self, factor_name, start_date, end_date, n_days=5, shared_data=None):
        """
        综合验证因子有效性
        高效优化版本：一次性加载所有数据到内存，避免重复IO操作，并简化验证流程
        
        Parameters:
        -----------
        factor_name : str
            因子名称
        start_date : str
            开始日期
        end_date : str
            结束日期
        n_days : int, optional
            未来收益计算天数
        shared_data : dict, optional
            已预加载的共享数据，如果提供则直接使用，避免重复加载
        
        Returns:
        --------
        dict
            加载的数据，可供其他因子验证重用
        """
        print(f"开始验证因子: {factor_name}")
        
        # 获取交易日序列
        trade_dates = self.analyzer.get_trade_dates(start_date, end_date)
        print(f"验证期间共有{len(trade_dates)}个交易日")
        
        # 创建数据容器
        all_data = {
            'factor_scores': {},      # 因子得分
            'future_returns': {},      # 未来收益率
            'market_returns': {},      # 市场收益率
            'sector_stats': {},        # 行业统计数据
            'quantiles': {},           # 分位数数据
            'stock_data': {},          # 股票数据
            'index_data': {}           # 指数数据
        }
        
        # 检查是否有共享数据，如果有则复用
        if shared_data and len(shared_data.get('stock_data', {})) > 0:
            print(f"使用共享数据进行验证，避免重复加载")
            # 复用共享数据中的基础数据
            for key in ['stock_data', 'future_returns', 'market_returns', 'sector_stats', 'quantiles', 'index_data']:
                if key in shared_data:
                    all_data[key] = shared_data[key]
            
            # 只需要重新计算当前因子的得分
            print("只需重新计算当前因子的得分...")
        else:
            # 使用批量数据处理方法一次性加载所有数据
            print("使用批量数据处理方法加载所有必要数据...")
            batch_data = self.batch_process_data(trade_dates, n_days)
            
            # 复制批量处理的数据
            for key in batch_data:
                all_data[key] = batch_data[key]
        
        # 计算所有日期的因子得分 - 这一步无论是否有共享数据都需要执行
        import concurrent.futures
        import threading
        
        # 创建线程锁，保护共享资源
        lock = threading.Lock()
        
        # 定义计算单个日期因子得分的函数
        def calculate_factor_score(date):
            try:
                if date in all_data['stock_data'] and date in all_data['sector_stats'] and date in all_data['quantiles']:
                    stock_data = all_data['stock_data'][date]
                    sector_stats = all_data['sector_stats'][date]
                    quantiles = all_data['quantiles'][date]
                    
                    # 计算因子得分
                    final_scores = self.analyzer.calculate_scores(sector_stats, quantiles, stock_data)
                    factor_scores = self.get_factor_scores(factor_name, final_scores)
                    return date, factor_scores
            except Exception as e:
                print(f"计算日期{date}的因子得分失败: {str(e)}")
            return date, None
        
        # 使用线程池并行计算因子得分
        print("并行计算所有日期的因子得分...")
        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
            futures = {executor.submit(calculate_factor_score, date): date for date in trade_dates}
            for future in concurrent.futures.as_completed(futures):
                date, factor_scores = future.result()
                if factor_scores is not None:
                    with lock:
                        all_data['factor_scores'][date] = factor_scores
        
        print(f"数据处理完成，共加载 {len(all_data['stock_data'])} 个交易日的股票数据")
        print(f"共加载 {len(all_data['future_returns'])} 个交易日的未来收益率")
        print(f"共加载 {len(all_data['factor_scores'])} 个交易日的因子得分")
        
        # 5. 计算IC值序列
        print(f"\n1. {factor_name} IC值序列分析")
        ic_values = {}
        valid_dates = []
        
        # 并行计算IC值
        def calculate_ic(date):
            if date in all_data['factor_scores'] and date in all_data['future_returns']:
                factor_scores = all_data['factor_scores'][date]
                future_returns = all_data['future_returns'][date]
                
                # 计算IC值
                ic = self.calculate_ic(factor_scores, future_returns)
                if ic is not None:
                    return date, ic
            return date, None
        
        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
            futures = {executor.submit(calculate_ic, date): date for date in trade_dates}
            for future in concurrent.futures.as_completed(futures):
                date, ic = future.result()
                if ic is not None:
                    with lock:
                        ic_values[date] = ic
                        valid_dates.append(date)
        
        # 转换为Series以便后续分析
        ic_series = pd.Series(ic_values)
        
        # 计算IC序列统计量
        if len(ic_series) > 0:
            ic_mean = ic_series.mean()
            ic_std = ic_series.std()
            ic_ir = ic_mean / ic_std if ic_std > 0 else np.nan
            ic_positive_ratio = (ic_series > 0).mean()
            
            print(f"{factor_name} IC均值: {ic_mean:.4f}")
            print(f"{factor_name} IC标准差: {ic_std:.4f}")
            print(f"{factor_name} IR比率: {ic_ir:.4f}")
            print(f"{factor_name} IC值为正的比例: {ic_positive_ratio:.2%}")
            
            # 进行t检验
            t_stat, p_value = self.run_t_test(ic_series)
            if t_stat is not None and p_value is not None:
                print(f"{factor_name} t统计量: {t_stat:.4f}")
                print(f"{factor_name} p值: {p_value:.4f}")
                print(f"{factor_name} 显著性: {'显著' if p_value < 0.05 else '不显著'}")
        else:
            print(f"{factor_name} 无法计算有效的IC序列")
        
        # 6. 分位数回测
        print(f"\n2. {factor_name} 分位数回测")
        cumulative_returns, backtest_dates = self.run_quantile_backtest(factor_name, start_date, end_date, 5, n_days, all_data)
        
        # 7. 参数敏感性测试
        print(f"\n3. {factor_name} 参数敏感性测试")
        param_values = [0.5, 0.8, 1.0, 1.2, 1.5] if factor_name != 'historical' else [50, 70, 80, 90, 95]
        avg_ic_values = self.run_parameter_sensitivity(factor_name, start_date, end_date, param_values, n_days, all_data)
        
        # 8. 市场状态测试
        print(f"\n4. {factor_name} 市场状态测试")
        market_state_ic = self.run_market_state_test(factor_name, start_date, end_date, n_days, all_data)
        
        return all_data
        
    def run_market_state_test(self, factor_name, start_date, end_date, n_days=5, all_data=None):
        """
        在不同市场状态下测试因子表现
        完全优化版本：使用内存中的数据进行计算，避免重复IO操作
        
        Parameters:
        -----------
        factor_name : str
            因子名称
        start_date : str
            开始日期
        end_date : str
            结束日期
        n_days : int
            未来收益计算天数
        all_data : dict
            预先加载的所有数据，如果为None则会重新加载
        """
        # 获取交易日序列
        trade_dates = self.analyzer.get_trade_dates(start_date, end_date)
        
        # 如果没有传入预加载的数据，则需要加载数据
        if all_data is None:
            print(f"没有传入预加载数据，开始加载市场状态测试所需数据...")
            all_data = {
                'factor_scores': {},
                'future_returns': {},
                'index_data': {}
            }
            
            # 检查是否有预加载的数据
            has_preloaded_data = len(self.cache_manager.preloaded_stock_data) > 0
            if not has_preloaded_data:
                print(f"预加载数据不存在，将加载所有必要数据")
                # 预加载所有必要的数据
                self.cache_manager.preload_stock_data(trade_dates)
                self.cache_manager.preload_future_returns(trade_dates, n_days)
                self.cache_manager.preload_index_data(trade_dates)
            else:
                print(f"使用已预加载的数据进行市场状态测试")
            
            # 在内存中计算所有日期的因子得分、未来收益率和指数数据
            print("正在内存中计算所有交易日的因子得分、未来收益率和指数数据...")
            for date in trade_dates:
                try:
                    # 获取股票数据
                    stock_data = self.cache_manager.get_preloaded_stock_data(date)
                    if stock_data is None:
                        stock_data = self.analyzer.get_stock_data(date)
                    
                    if stock_data is not None and not stock_data.empty:
                        # 计算因子得分
                        sector_stats, quantiles = self.analyzer.calculate_indicators(date)
                        final_scores = self.analyzer.calculate_scores(sector_stats, quantiles, stock_data)
                        
                        # 获取特定因子得分
                        factor_scores = self.get_factor_scores(factor_name, final_scores)
                        all_data['factor_scores'][date] = factor_scores
                    
                    # 获取未来收益率
                    future_returns = self.cache_manager.get_preloaded_future_returns(date, n_days)
                    if future_returns is None:
                        future_returns = self.get_future_returns(date, n_days)
                    if future_returns is not None:
                        all_data['future_returns'][date] = future_returns
                    
                    # 获取指数数据
                    index_data = self.cache_manager.get_preloaded_index_data(date)
                    if index_data is None:
                        index_data = self.analyzer.get_index_data(date)
                    if index_data is not None and not index_data.empty:
                        all_data['index_data'][date] = index_data.iloc[0]
                except Exception as e:
                    print(f"处理日期{date}数据失败: {str(e)}")
                    continue
            
            print(f"数据处理完成，成功加载 {len(all_data['factor_scores'])} 个交易日的因子得分")
            print(f"成功加载 {len(all_data['future_returns'])} 个交易日的未来收益率")
            print(f"成功加载 {len(all_data['index_data'])} 个交易日的指数数据")
        else:
            print(f"使用传入的预加载数据进行市场状态测试，共有 {len(all_data['factor_scores'])} 个交易日的数据")
        
        # 构建指数数据DataFrame
        if 'index_data' not in all_data or not all_data['index_data']:
            print("没有有效的指数数据，无法进行市场状态测试")
            return None
        
        # 将字典转换为DataFrame
        index_df = pd.DataFrame(all_data['index_data']).T
        index_df['close'] = index_df['close'].astype(float)
        
        # 计算20日收益率，用于判断市场状态
        index_df['return_20d'] = index_df['close'].pct_change(20)
        
        # 定义市场状态：牛市、熊市、震荡市
        # 简化处理：20日收益率>5%为牛市，<-5%为熊市，其他为震荡市
        index_df['market_state'] = 'flat'
        index_df.loc[index_df['return_20d'] > 0.05, 'market_state'] = 'bull'
        index_df.loc[index_df['return_20d'] < -0.05, 'market_state'] = 'bear'
        
        # 初始化不同市场状态下的IC值列表
        market_state_ic = {'bull': [], 'bear': [], 'flat': []}
        
        # 使用内存中的数据进行市场状态测试
        for date in trade_dates:
            try:
                if date in all_data['factor_scores'] and date in all_data['future_returns'] and date in index_df.index:
                    factor_scores = all_data['factor_scores'][date]
                    future_returns = all_data['future_returns'][date]
                    market_state = index_df.loc[date, 'market_state']
                    
                    # 计算IC值
                    ic = self.calculate_ic(factor_scores, future_returns)
                    if ic is not None:
                        market_state_ic[market_state].append(ic)
            except Exception as e:
                print(f"日期{date}市场状态测试失败: {str(e)}")
                continue
        
        # 计算各市场状态下的平均IC值
        avg_market_state_ic = {state: np.mean(ics) if ics else np.nan for state, ics in market_state_ic.items()}
        
        return avg_market_state_ic


    def run_risk_adjusted_returns(self, factor_name, start_date, end_date, n_days, all_data=None):
        """
        计算因子的风险调整收益指标，包括最大回撤、夏普比率、Alpha和Beta
        """
        logger = logging.getLogger(__name__)
        logger.info(f"开始计算{factor_name}的风险调整收益指标...")
        
        try:
            # 获取交易日列表
            trade_dates = self.analyzer.get_trade_dates(start_date, end_date)
            if not trade_dates:
                return None
            
            # 初始化结果字典
            results = {
                'high_cumulative': [],
                'low_cumulative': [],
                'market_cumulative': [],
                'max_drawdown': None,
                'sharpe_ratio': None,
                'alpha': None,
                'beta': None
            }
            
            # 获取每个交易日的因子得分和未来收益
            high_group_returns = []
            low_group_returns = []
            market_returns = []
            
            for date in trade_dates:
                # 获取因子得分
                factor_scores = self.analyzer.calculate_factor_scores(factor_name, date)
                if factor_scores is None:
                    continue
                    
                # 获取未来收益
                future_returns = self.get_future_returns(date, n_days)
                if future_returns is None:
                    continue
                
                # 将因子得分排序，选择高分组和低分组
                sorted_scores = pd.Series(factor_scores).sort_values(ascending=False)
                high_group = sorted_scores.head(len(sorted_scores) // 5).index  # 前20%
                low_group = sorted_scores.tail(len(sorted_scores) // 5).index   # 后20%
                
                # 计算分组收益
                high_return = future_returns[high_group].mean()
                low_return = future_returns[low_group].mean()
                market_return = future_returns.mean()
                
                high_group_returns.append(high_return)
                low_group_returns.append(low_return)
                market_returns.append(market_return)
            
            # 转换为numpy数组
            high_returns = np.array(high_group_returns)
            low_returns = np.array(low_group_returns)
            market_returns = np.array(market_returns)
            
            # 计算累积收益
            high_cumulative = np.cumprod(1 + high_returns) - 1
            low_cumulative = np.cumprod(1 + low_returns) - 1
            market_cumulative = np.cumprod(1 + market_returns) - 1
            
            results['high_cumulative'] = high_cumulative
            results['low_cumulative'] = low_cumulative
            results['market_cumulative'] = market_cumulative
            
            # 计算最大回撤
            def calculate_max_drawdown(returns):
                cumulative = np.cumprod(1 + returns)
                running_max = np.maximum.accumulate(cumulative)
                drawdown = (cumulative - running_max) / running_max
                return np.min(drawdown)
            
            results['max_drawdown'] = calculate_max_drawdown(high_returns)
            
            # 计算夏普比率（假设无风险利率为3%）
            risk_free_rate = 0.03 / 252  # 日化无风险利率
            excess_returns = high_returns - risk_free_rate
            if len(excess_returns) > 0:
                sharpe_ratio = np.sqrt(252) * np.mean(excess_returns) / np.std(excess_returns)
                results['sharpe_ratio'] = sharpe_ratio
            
            # 计算Alpha和Beta
            if len(high_returns) > 0 and len(market_returns) > 0:
                # 使用市场模型回归
                X = sm.add_constant(market_returns)
                model = sm.OLS(high_returns, X).fit()
                
                results['alpha'] = model.params[0] * 252  # 年化Alpha
                results['beta'] = model.params[1]        # Beta
            
            return results
            
        except Exception as e:
            logger.error(f"计算风险调整收益指标时发生错误: {str(e)}")
            return None